package com.access;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.data.JDoc;

public class Crawler {

	// data for inserting into db
	private Map<String, Integer> urlToDocId;
	private Map<Integer, JDoc> idToDoc;
	private Map<String, Map<Integer, Float>> invertedIndex;

	// data for stemming
	private Map<Integer, ArrayList<String>> docContent;
	private ArrayList<String> content;

	// constants
	private int limit;
	private int counter;

	// constructor
	public Crawler() {
		urlToDocId = new HashMap<String, Integer>();
		idToDoc = new HashMap<Integer, JDoc>();
		docContent = new HashMap<Integer, ArrayList<String>>();
	}

	// start crawling
	public void crawl(String root, int numOfPages) {

		limit = numOfPages;
		ArrayList<String> queue = new ArrayList<String>();
		queue.add(root);
		limit--;

		counter = 1;
		System.out.println("start crawling");

		searchUrl(queue, null);

		if (docContent == null || docContent.size() <= 0) {
			System.err.println("\nErrors found in crawling\n");
			System.exit(0);
		}

		System.out.println("\nfinish crawling");

		try {
			Stemmer stemmer = new Stemmer();
			stemmer.stemAll(docContent);

			invertedIndex = stemmer.getInvertedIndex();

			Map<Integer, Map<String, Integer>> stemmedContent = stemmer
					.getStemmedContent();
			for (Integer docId : stemmedContent.keySet()) {
				idToDoc.get(docId).setWords(stemmedContent.get(docId));
			}

		} catch (FileNotFoundException e) {
			e.printStackTrace();
			System.err.println("\nError found in initializing stemmer\n");
			System.exit(0);
		}

		System.out.println("\nfinish stemming");
	}

	// search all url from the root url recursively using breath first search
	private void searchUrl(ArrayList<String> queue, String parent) {

		String currentNode;
		JDoc doc = null;
		ArrayList<String> children = null;

		if (queue != null && queue.size() > 0) {
			currentNode = queue.remove(0); // pop from the queue

			try {
				System.out.print("start building doc " + counter + " ...");
				doc = buildDoc(currentNode);
				System.out.print(" finish building doc" + counter + " ...\n");
				counter++;

				children = doc.getChildLinks();
				doc.addParent(parent);

				// data for stemming
				docContent.put(idToDoc.size(), content);

				// data for inserting to db
				urlToDocId.put(currentNode, urlToDocId.size());
				idToDoc.put(idToDoc.size(), doc);

			} catch (Exception e) {
				System.err.println("\n" + e + "\n");
				currentNode = parent;
			}
		} else {
			return;
		}

		if (children != null && children.size() > 0) {
			for (String child : children) {
				if (!urlToDocId.containsKey(child)) {
					// children that have not been created
					if (limit <= 0) {
						break;
					}
					queue.add(child);
					limit--;
				} else {
					// children that have been created
					idToDoc.get(urlToDocId.get(child)).addParent(currentNode);
				}
			}
		}
		searchUrl(queue, currentNode);
	}

	// retrieve data from given url, build and return corresponding doc
	private JDoc buildDoc(String url) throws IOException {

		// data for building doc
		String title;
		String lastModified;
		String body;
		ArrayList<String> childLinks = new ArrayList<String>();
		ArrayList<String> stemmedTitle = new ArrayList<String>();
		ArrayList<String> stemmedBody = new ArrayList<String>();

		Extractor extractor = new Extractor(url);
		title = extractor.getTitle();
		lastModified = extractor.getLastModified();
		childLinks = extractor.getChildLinks();
		body = extractor.getContent();

		// // find title
		// title = Extractor.extractTitle(url);
		//
		// // find last modified date
		// lastModified = Extractor.extractLastModifiedDate(url);
		//
		// // find child links
		// childLinks = Extractor.extractChildLinks(url);
		//
		// // find the content
		// String body = Extractor.extractContent(url);

		Stemmer stemmer = new Stemmer();
		stemmedTitle = stemmer.stemWithRepeat(title);
		stemmedBody = stemmer.stemWithRepeat(body);

		// build the doc and prepare words list for stemming
		if (!body.isEmpty()) {
			// split the content
			List<String> wordList = Arrays.asList(body.split("[^a-zA-Z0-9]"));

			if (wordList != null && wordList.size() > 0) {
				ArrayList<String> wordArrayList = new ArrayList<String>(
						wordList);
				// remove white space
				wordArrayList.removeAll(Arrays.asList("", null));

				if (wordArrayList != null && wordArrayList.size() > 0) {
					content = wordArrayList;
					return new JDoc(url, title, lastModified, 0, childLinks,
							stemmedTitle, stemmedBody);
				}
			}
		}

		content = null;
		return new JDoc(url, title, lastModified, 0, childLinks, stemmedTitle,
				stemmedBody);

	}

	public Map<String, Integer> getUrlToDocId() {
		return urlToDocId;
	}

	public Map<Integer, JDoc> getIdToDoc() {
		return idToDoc;
	}

	public Map<String, Map<Integer, Float>> getInvertedIndex() {
		return invertedIndex;
	}

}
